{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6789eab6-9237-41f7-8323-9f0cd0d6e3d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============GPU================\n",
      "Thu May  9 02:23:00 2024       \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 550.54.15              Driver Version: 550.54.15      CUDA Version: 12.4     |\n",
      "|-----------------------------------------+------------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                        |               MIG M. |\n",
      "|=========================================+========================+======================|\n",
      "|   0  NVIDIA GeForce RTX 3080        On  |   00000000:26:00.0 Off |                  N/A |\n",
      "| 72%   69C    P0            320W /  380W |       1MiB /  10240MiB |     72%      Default |\n",
      "|                                         |                        |                  N/A |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "                                                                                         \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                              |\n",
      "|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |\n",
      "|        ID   ID                                                               Usage      |\n",
      "|=========================================================================================|\n",
      "|  No running processes found                                                             |\n",
      "+-----------------------------------------------------------------------------------------+\n",
      "============CUDA version================\n",
      "nvcc: NVIDIA (R) Cuda compiler driver\n",
      "Copyright (c) 2005-2023 NVIDIA Corporation\n",
      "Built on Mon_Apr__3_17:16:06_PDT_2023\n",
      "Cuda compilation tools, release 12.1, V12.1.105\n",
      "Build cuda_12.1.r12.1/compiler.32688072_0\n",
      "============CPU================\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "model name\t: AMD Ryzen 5 3600X 6-Core Processor\n",
      "============Memory================\n",
      "MemTotal:       16289820 kB\n"
     ]
    }
   ],
   "source": [
    "# GPU\n",
    "print(\"============GPU================\")\n",
    "!nvidia-smi\n",
    "\n",
    "# CUDA version\n",
    "print(\"============CUDA version================\")\n",
    "!nvcc --version\n",
    "\n",
    "# CPU\n",
    "print(\"============CPU================\")\n",
    "!cat /proc/cpuinfo | grep model\\ name\n",
    "\n",
    "# Memory\n",
    "print(\"============Memory================\")\n",
    "!cat /proc/meminfo | grep MemTotal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9583f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'llama.cpp'...\n",
      "remote: Enumerating objects: 24130, done.\u001b[K\n",
      "remote: Counting objects: 100% (9002/9002), done.\u001b[K\n",
      "remote: Compressing objects: 100% (534/534), done.\u001b[K\n",
      "remote: Total 24130 (delta 8724), reused 8526 (delta 8467), pack-reused 15128\u001b[K\n",
      "Receiving objects: 100% (24130/24130), 45.90 MiB | 14.03 MiB/s, done.\n",
      "Resolving deltas: 100% (17143/17143), done.\n",
      "/workspace/llama.cpp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "# Get the code for the first time use\n",
    "!git clone https://github.com/ggerganov/llama.cpp\n",
    "%cd llama.cpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f651103-a6c8-4b7d-9f0f-be0553b22acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ggml-vocab-aquila.gguf\t\t    ggml-vocab-llama-bpe.gguf\n",
      "ggml-vocab-baichuan.gguf\t    ggml-vocab-llama-bpe.gguf.inp\n",
      "ggml-vocab-bert-bge.gguf\t    ggml-vocab-llama-bpe.gguf.out\n",
      "ggml-vocab-bert-bge.gguf.inp\t    ggml-vocab-llama-spm.gguf\n",
      "ggml-vocab-bert-bge.gguf.out\t    ggml-vocab-llama-spm.gguf.inp\n",
      "ggml-vocab-command-r.gguf\t    ggml-vocab-llama-spm.gguf.out\n",
      "ggml-vocab-command-r.gguf.inp\t    ggml-vocab-mpt.gguf\n",
      "ggml-vocab-command-r.gguf.out\t    ggml-vocab-mpt.gguf.inp\n",
      "ggml-vocab-deepseek-coder.gguf\t    ggml-vocab-mpt.gguf.out\n",
      "ggml-vocab-deepseek-coder.gguf.inp  ggml-vocab-phi-3.gguf\n",
      "ggml-vocab-deepseek-coder.gguf.out  ggml-vocab-phi-3.gguf.inp\n",
      "ggml-vocab-deepseek-llm.gguf\t    ggml-vocab-phi-3.gguf.out\n",
      "ggml-vocab-deepseek-llm.gguf.inp    ggml-vocab-qwen2.gguf\n",
      "ggml-vocab-deepseek-llm.gguf.out    ggml-vocab-qwen2.gguf.inp\n",
      "ggml-vocab-falcon.gguf\t\t    ggml-vocab-qwen2.gguf.out\n",
      "ggml-vocab-falcon.gguf.inp\t    ggml-vocab-refact.gguf\n",
      "ggml-vocab-falcon.gguf.out\t    ggml-vocab-refact.gguf.inp\n",
      "ggml-vocab-gpt-2.gguf\t\t    ggml-vocab-refact.gguf.out\n",
      "ggml-vocab-gpt-2.gguf.inp\t    ggml-vocab-stablelm.gguf\n",
      "ggml-vocab-gpt-2.gguf.out\t    ggml-vocab-starcoder.gguf\n",
      "ggml-vocab-gpt-neox.gguf\t    ggml-vocab-starcoder.gguf.inp\n",
      "ggml-vocab-gpt2.gguf\t\t    ggml-vocab-starcoder.gguf.out\n"
     ]
    }
   ],
   "source": [
    "# Obtain the original LLaMA model weights and place them in ./models\n",
    "!ls ./models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "58f434b3-1493-448d-95d3-8954c1aa1267",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get:1 http://archive.ubuntu.com/ubuntu jammy InRelease [270 kB]\n",
      "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease [1581 B]\n",
      "Get:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  Packages [830 kB]\n",
      "Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]      \u001b[0m\n",
      "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]        \u001b[0m\u001b[33m\u001b[33m\n",
      "Get:6 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB]      \u001b[0m\u001b[33m\u001b[33m\n",
      "Get:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]3m\n",
      "Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 Packages [17.5 MB] \u001b[0m\u001b[33m\u001b[33m\n",
      "Get:9 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [27.8 kB]\n",
      "Get:10 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1789 kB]33m\n",
      "Get:11 http://archive.ubuntu.com/ubuntu jammy/restricted amd64 Packages [164 kB]\n",
      "Get:12 http://archive.ubuntu.com/ubuntu jammy/multiverse amd64 Packages [266 kB][0m\u001b[33m\n",
      "Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 Packages [1792 kB]    \u001b[0m\u001b[33m\n",
      "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [51.1 kB]\n",
      "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2382 kB]\n",
      "Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1373 kB]m\n",
      "Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2069 kB]\u001b[33m\n",
      "Get:18 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [81.0 kB]33m\u001b[33m\n",
      "Get:19 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [31.9 kB]\n",
      "Get:20 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1078 kB]\n",
      "Get:21 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2308 kB]\n",
      "Get:22 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [44.7 kB]\n",
      "Fetched 32.4 MB in 3s (12.8 MB/s)[33m                       \u001b[0m\u001b[33m\u001b[33m\u001b[33m\u001b[33m\u001b[33m\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "56 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "The following NEW packages will be installed:\n",
      "  git-lfs\n",
      "0 upgraded, 1 newly installed, 0 to remove and 56 not upgraded.\n",
      "Need to get 3503 kB of archives.\n",
      "After this operation, 10.4 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 git-lfs amd64 3.0.2-1ubuntu0.2 [3503 kB]\n",
      "Fetched 3503 kB in 1s (3654 kB/s) \u001b[0m[33m\u001b[33m\n",
      "debconf: delaying package configuration, since apt-utils is not installed\n",
      "\n",
      "\u001b7\u001b[0;23r\u001b8\u001b[1ASelecting previously unselected package git-lfs.\n",
      "(Reading database ... 21038 files and directories currently installed.)\n",
      "Preparing to unpack .../git-lfs_3.0.2-1ubuntu0.2_amd64.deb ...\n",
      "\u001b7\u001b[24;0f\u001b[42m\u001b[30mProgress: [  0%]\u001b[49m\u001b[39m [..........................................................] \u001b8\u001b7\u001b[24;0f\u001b[42m\u001b[30mProgress: [ 20%]\u001b[49m\u001b[39m [###########...............................................] \u001b8Unpacking git-lfs (3.0.2-1ubuntu0.2) ...\n",
      "\u001b7\u001b[24;0f\u001b[42m\u001b[30mProgress: [ 40%]\u001b[49m\u001b[39m [#######################...................................] \u001b8Setting up git-lfs (3.0.2-1ubuntu0.2) ...\n",
      "\u001b7\u001b[24;0f\u001b[42m\u001b[30mProgress: [ 60%]\u001b[49m\u001b[39m [##################################........................] \u001b8\u001b7\u001b[24;0f\u001b[42m\u001b[30mProgress: [ 80%]\u001b[49m\u001b[39m [##############################################............] \u001b8\n",
      "\u001b7\u001b[0;24r\u001b8\u001b[1A\u001b[J"
     ]
    }
   ],
   "source": [
    "# Get git-lfs to clone large files\n",
    "!apt update\n",
    "!apt install git-lfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e63016a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp/models\n",
      "Updated git hooks.\n",
      "Git LFS initialized.\n",
      "Cloning into 'Llama-3-8B-GGUF'...\n",
      "remote: Enumerating objects: 44, done.\u001b[K\n",
      "remote: Counting objects: 100% (41/41), done.\u001b[K\n",
      "remote: Compressing objects: 100% (41/41), done.\u001b[K\n",
      "remote: Total 44 (delta 9), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
      "Unpacking objects: 100% (44/44), 2.25 MiB | 3.25 MiB/s, done.\n",
      "Filtering content:  87% (7/8), 3.54 GiB | 3.53 MiB/s\r"
     ]
    }
   ],
   "source": [
    "# Clone/Download the model files from Meta HF repo: https://huggingface.co/meta-llama. Or feel free to clone from my HF repo:\n",
    "%cd models\n",
    "!git lfs install\n",
    "!git clone https://huggingface.co/JaaackXD/Llama-3-8B-GGUF\n",
    "# !git clone https://huggingface.co/JaaackXD/Llama-3-70B-GGUF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "603a64eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir 8B-v3\n",
    "!mv Llama-3-8B-GGUF/*.gguf 8B-v3\n",
    "# !mkdir 70B-v3\n",
    "# !mv Llama-3-70B-GGUF/*.gguf 70B-v3\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43e9d80e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build\n",
    "!make clean && LLAMA_CUBLAS=1 make -j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ca0e26e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the code if you would like to convert and quantize models by yourself\n",
    "# Install Python dependencies\n",
    "!python3 -m pip install -r requirements.txt\n",
    "\n",
    "# # Convert the HF models to ggml FP16 format (High RAM requirement!)\n",
    "# !python3 convert-hf-to-gguf.py models/Llama-3-8B-GGUF/Meta-Llama-3-8B/ --outfile models/8B-v3/ggml-model-f16.gguf --outtype f16\n",
    "# !python3 convert-hf-to-gguf.py models/Llama-3-70B-GGUF/Meta-Llama-3-70B/ --outfile models/70B-v3/ggml-model-f16.gguf --outtype f16\n",
    "\n",
    "# # Quantize the model to 4-bits (using Q4_K_M method)\n",
    "# !./quantize ./models/8B-v3/ggml-model-f16.gguf ./models/8B-v3/ggml-model-Q4_K_M.gguf Q4_K_M\n",
    "# !./quantize ./models/70B-v3/ggml-model-f16.gguf ./models/70B-v3/ggml-model-Q4_K_M.gguf Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "30466fa3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 2824 (4426e298)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/8B-v3/ggml-model-Q4_K_M.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = Meta-Llama-3-8B\n",
      "llama_model_loader: - kv   2:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   3:                       llama.context_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 14336\n",
      "llama_model_loader: - kv   6:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   7:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   8:                       llama.rope.freq_base f32              = 500000.000000\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 15\n",
      "llama_model_loader: - kv  11:                           llama.vocab_size u32              = 128256\n",
      "llama_model_loader: - kv  12:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv  13:                       tokenizer.ggml.model str              = gpt2\n",
      "llama_model_loader: - kv  14:                         tokenizer.ggml.pre str              = llama-bpe\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,128256]  = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
      "llama_model_loader: - kv  16:                  tokenizer.ggml.token_type arr[i32,128256]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  17:                      tokenizer.ggml.merges arr[str,280147]  = [\"Ġ Ġ\", \"Ġ ĠĠĠ\", \"ĠĠ ĠĠ\", \"...\n",
      "llama_model_loader: - kv  18:                tokenizer.ggml.bos_token_id u32              = 128000\n",
      "llama_model_loader: - kv  19:                tokenizer.ggml.eos_token_id u32              = 128001\n",
      "llama_model_loader: - kv  20:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_K:  193 tensors\n",
      "llama_model_loader: - type q6_K:   33 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 256/128256 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = BPE\n",
      "llm_load_print_meta: n_vocab          = 128256\n",
      "llm_load_print_meta: n_merges         = 280147\n",
      "llm_load_print_meta: n_ctx_train      = 8192\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_embd_head_k    = 128\n",
      "llm_load_print_meta: n_embd_head_v    = 128\n",
      "llm_load_print_meta: n_gqa            = 4\n",
      "llm_load_print_meta: n_embd_k_gqa     = 1024\n",
      "llm_load_print_meta: n_embd_v_gqa     = 1024\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 14336\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: causal attn      = 1\n",
      "llm_load_print_meta: pooling type     = 0\n",
      "llm_load_print_meta: rope type        = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 500000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 8192\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: ssm_d_conv       = 0\n",
      "llm_load_print_meta: ssm_d_inner      = 0\n",
      "llm_load_print_meta: ssm_d_state      = 0\n",
      "llm_load_print_meta: ssm_dt_rank      = 0\n",
      "llm_load_print_meta: model type       = 8B\n",
      "llm_load_print_meta: model ftype      = Q4_K - Medium\n",
      "llm_load_print_meta: model params     = 8.03 B\n",
      "llm_load_print_meta: model size       = 4.58 GiB (4.89 BPW) \n",
      "llm_load_print_meta: general.name     = Meta-Llama-3-8B\n",
      "llm_load_print_meta: BOS token        = 128000 '<|begin_of_text|>'\n",
      "llm_load_print_meta: EOS token        = 128001 '<|end_of_text|>'\n",
      "llm_load_print_meta: LF token         = 128 'Ä'\n",
      "llm_load_print_meta: EOT token        = 128009 '<|eot_id|>'\n",
      "ggml_cuda_init: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_cuda_init: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3080, compute capability 8.6, VMM: yes\n",
      "llm_load_tensors: ggml ctx size =    0.30 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "llm_load_tensors:        CPU buffer size =   281.81 MiB\n",
      "llm_load_tensors:      CUDA0 buffer size =  4403.49 MiB\n",
      "........................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 8192\n",
      "llama_new_context_with_model: n_batch    = 2048\n",
      "llama_new_context_with_model: n_ubatch   = 512\n",
      "llama_new_context_with_model: flash_attn = 0\n",
      "llama_new_context_with_model: freq_base  = 500000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init:      CUDA0 KV buffer size =  1024.00 MiB\n",
      "llama_new_context_with_model: KV self size  = 1024.00 MiB, K (f16):  512.00 MiB, V (f16):  512.00 MiB\n",
      "llama_new_context_with_model:  CUDA_Host  output buffer size =     0.49 MiB\n",
      "llama_new_context_with_model:      CUDA0 compute buffer size =   560.00 MiB\n",
      "llama_new_context_with_model:  CUDA_Host compute buffer size =    24.01 MiB\n",
      "llama_new_context_with_model: graph nodes  = 1030\n",
      "llama_new_context_with_model: graph splits = 2\n",
      "\n",
      "system_info: n_threads = 6 / 12 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temperature \n",
      "generate: n_ctx = 8192, n_batch = 2048, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m<|begin_of_text|> First Citizen:\n",
      "\n",
      " Before we proceed any further, hear me speak.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Speak, speak.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " You are all resolved rather to die than to famish?\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Resolved. resolved.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " First, you know Caius Marcius is chief enemy to the people.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " We know't, we know't.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " Let us kill him, and we'll have corn at our own price. Is't a verdict?\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " No more talking on't; let it be done: away, away!\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " One word, good citizens.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " We are accounted poor citizens, the patricians good. What authority surfeits on would relieve us: if they would yield us but the superfluity,  while it were wholesome, we might guess they relieved us humanely; but they think we are too dear: the leanness that afflicts us, the object of  our misery, is as an inventory to particularise their abundance; our sufferance is a gain to them Let us revenge this with our pikes,  ere we become rakes: for the gods know I speak this in hunger for bread, not in thirst for revenge.\n",
      "\n",
      " \n",
      "\n",
      " \u001b[0m First Citizen:\n",
      "\n",
      " Do you hear, hear?\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " What would you have me to do? throw your eyes\n",
      "\n",
      " As in my cause they should be forced by me;\n",
      "\n",
      " The grievance of the whole body, let it be called,\n",
      "\n",
      " Should be as fresh to me as it was then: but I doubt their prosperity\n",
      "\n",
      " Will be in very ease; yet my condition  Is both for I and friendly, and for our safety.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Therefore bear patience,  For we have mutual help and mutual love, Beside, we are men sharing a common light.   I will not fail.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " We'll bond ourselves to't,  And as the spirit lifts us from below Were twain that were imbrac'd and twins of love Then let them fight by fair encounter.\n",
      "\n",
      " \n",
      "\n",
      " First Citizen:\n",
      "\n",
      " You have well sinned against yourself in thriving\n",
      "\n",
      " The city must have entrance or we suffocate.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " Come; get you to the Capitol; lay hold upon Caius Martius ere he do open the gates.\n",
      "\n",
      "  First Citizen:\n",
      "\n",
      " He is no free man,  Since that thy moody flats to heaven render scornfully: 'Tis he you would be ruled by, not that rules you; therefore you shall forgive him duty, if you would put him out of pain\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " That's good. That will I.\n",
      "\n",
      " \n",
      "\n",
      " Second Citizen:\n",
      "\n",
      " Pray let us have some retirement:  there is in this man An absolute and perfect contrariety to custom; And then to face the authority of eyes, Exce'd all use of civil courtesy.   Come, bring your luggage, nay, bring it with you, let's go: I will not tarry; no, nor against This cholic that our city hath in't,  The heart is quench'd, and enromag'd up only Want, wanting the service of a more penurious  place, to lie with his pack and let him need'st your petition.   Come.\n",
      "\n",
      " \n",
      "\n",
      " All:\n",
      "\n",
      " We'll use this ungracious pause; make answer to your suit.  [Exeunt.\n",
      "\n",
      " <|end_of_text|> [end of text]\n",
      "\n",
      "llama_print_timings:        load time =     799.23 ms\n",
      "llama_print_timings:      sample time =     442.27 ms /   432 runs   (    1.02 ms per token,   976.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =      98.40 ms /   258 tokens (    0.38 ms per token,  2621.87 tokens per second)\n",
      "llama_print_timings:        eval time =    4059.17 ms /   431 runs   (    9.42 ms per token,   106.18 tokens per second)\n",
      "llama_print_timings:       total time =    4716.96 ms /   689 tokens\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# Start inference on a gguf model (-h to show all options)\n",
    "!./main -ngl 10000 -m ./models/8B-v3/ggml-model-Q4_K_M.gguf --color --temp 1.1 --repeat_penalty 1.1 -c 0 -n 1024 -e -s 0 -p \"\"\"\\\n",
    "First Citizen:\\n\\n\\\n",
    "Before we proceed any further, hear me speak.\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "Speak, speak.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "You are all resolved rather to die than to famish?\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "Resolved. resolved.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "First, you know Caius Marcius is chief enemy to the people.\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "We know't, we know't.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "Let us kill him, and we'll have corn at our own price. Is't a verdict?\\n\\n\\\n",
    "\\n\\n\\\n",
    "All:\\n\\n\\\n",
    "No more talking on't; let it be done: away, away!\\n\\n\\\n",
    "\\n\\n\\\n",
    "Second Citizen:\\n\\n\\\n",
    "One word, good citizens.\\n\\n\\\n",
    "\\n\\n\\\n",
    "First Citizen:\\n\\n\\\n",
    "We are accounted poor citizens, the patricians good. What authority surfeits on would relieve us: if they would yield us but the superfluity, \\\n",
    "while it were wholesome, we might guess they relieved us humanely; but they think we are too dear: the leanness that afflicts us, the object of \\\n",
    "our misery, is as an inventory to particularise their abundance; our sufferance is a gain to them Let us revenge this with our pikes, \\\n",
    "ere we become rakes: for the gods know I speak this in hunger for bread, not in thirst for revenge.\\n\\n\\\n",
    "\\n\\n\\\n",
    "\"\"\"\n",
    "\n",
    "# # Chat template for Termianl (Use the instruction-tuned models to better follow the template)\n",
    "# !./main -ngl 10000 -m ./models/8B-v3-instruct/ggml-model-Q4_K_M.gguf --color -c 0 -n -2 -e -s 0 --mirostat 2 -i --no-display-prompt --keep -1 \\\n",
    "# -r '<|eot_id|>' -p '<|begin_of_text|><|start_header_id|>system<|end_header_id|>\\n\\nYou are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\\n\\nHi!<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n' \\\n",
    "# --in-prefix '<|start_header_id|>user<|end_header_id|>\\n\\n' --in-suffix '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17c21608",
   "metadata": {},
   "source": [
    "## Benchmarks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6aee37",
   "metadata": {},
   "source": [
    "### 8B Q4_K_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "996ee79e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ggml_cuda_init: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_cuda_init: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3080, compute capability 8.6, VMM: yes\n",
      "| model                          |       size |     params | backend    | ngl | test       |              t/s |\n",
      "| ------------------------------ | ---------: | ---------: | ---------- | --: | ---------- | ---------------: |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | pp 512     |   3728.86 ± 4.77 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | pp 1024    |   3557.02 ± 2.95 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | pp 4096    |   2852.06 ± 0.31 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | pp 8192    |   2232.21 ± 2.66 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | tg 512     |    109.57 ± 0.19 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | tg 1024    |    106.40 ± 0.16 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | tg 4096    |     98.67 ± 0.05 |\n",
      "| llama 8B Q4_K - Medium         |   4.58 GiB |     8.03 B | CUDA       |  99 | tg 8192    |     89.90 ± 0.02 |\n",
      "\n",
      "build: 4426e298 (2824)\n"
     ]
    }
   ],
   "source": [
    "!./llama-bench -p 512,1024,4096,8192 -n 512,1024,4096,8192 -m ./models/8B-v3/ggml-model-Q4_K_M.gguf"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
